package edu.ptit.jad.preprocess.app;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.StringTokenizer;
import java.util.TreeSet;

import edu.ptit.jad.configuration.CONSTANTS;
import edu.ptit.jad.data.DataAccess;
import edu.ptit.jad.exception.AdException;

public class AppGenerateVob {

	// public static Set<String> allTermsSet = new HashSet<String>();

	static SortedSet<String> sortedset = new TreeSet<String>();
	static Map<String, Integer> termFrequencyMap = new HashMap<String, Integer>();
	static List<String> stopWords;

	public static void main(String[] args) throws AdException {

		System.out.println("stopwords loading....");
		stopWords = DataAccess.getInstance().getStopWords();
		System.out.println("news and ads processing....");

		//processFileDir(CONSTANTS.FILE_DATA_ADS_PRE);

		 processFileDir(CONSTANTS.FILE_DATA_NEWS_PRE);
		System.out.println("vobs are saving......");

		saveVocabs(CONSTANTS.FILE_VOBS);
		saveVocabFreqs("data/dict/vn_vobs_freq.jad");

		System.out.println("completed!");

	}

	public static void saveVocabs(String fileDir) throws AdException {

		try {
			File file = new File(fileDir);
			BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
					new FileOutputStream(file), "UTF8"));

			Iterator<String> it = sortedset.iterator();
			String term = "";
			while (it.hasNext()) {

				term = it.next();
				if (!stopWords.contains(term) && term != "") {
					out.write(term);
					out.write("\n");
				}

			}

		} catch (Exception e) {

			e.printStackTrace();
			throw new AdException("File write error " + fileDir);
		}
	}

	public static void saveVocabFreqs(String fileDir) throws AdException {

		try {
			File file = new File(fileDir);
			BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
					new FileOutputStream(file), "UTF8"));
			
			
			List<Map.Entry<String, Integer>> terms = new ArrayList<Map.Entry<String, Integer>>(
					termFrequencyMap.entrySet());

			// Different terms can have the same frequency.

			Collections.sort(terms, new Comparator<Map.Entry<String, Integer>>() {
				public int compare(Map.Entry<String, Integer> e1,
						Map.Entry<String, Integer> e2) {
					int result = 0;

					if (e1.getValue() < e2.getValue()) {
						result = 1; // reverse order
					} else if (e1.getValue() > e2.getValue()) {
						result = -1;
					} else {
						result = 0;
					}

					return result;
				}
			});
			

			for (Map.Entry<String, Integer> e : terms) {

				out.write(e.getKey() + " " + e.getValue());
				out.write("\n");
			}

		} catch (Exception e) {

			e.printStackTrace();
			throw new AdException("File write error " + fileDir);
		}
	}

	public static void processFileDir(String fileDir) {

		// Read read file

		try {
			BufferedReader reader = new BufferedReader(new InputStreamReader(
					new FileInputStream(fileDir), "UTF-8"));
			String line;

			// read the number of words
			line = reader.readLine();
			int nwords = Integer.parseInt(line);

			String term = "";
			// read map
			for (int i = 0; i < nwords; ++i) {
				line = reader.readLine();
				StringTokenizer tknr = new StringTokenizer(line, " ");

				while (tknr.hasMoreTokens()) {

					// allTermsSet.add(tknr.nextToken());
					// sortedset.add(tknr.nextToken());

					term = tknr.nextToken();
					
					if(containsDigit(term)) continue;

					if (!stopWords.contains(term) && term != "") {

						sortedset.add(term);
						Integer frequency = termFrequencyMap.get(term);
						if (frequency == null) {
							termFrequencyMap.put(term, 1);
						} else {
							termFrequencyMap.put(term, frequency + 1);
						}

					}

				}

			}

			reader.close();

		} catch (Exception e) {
			System.out.println("Error while reading dictionary:"
					+ e.getMessage());
			e.printStackTrace();

		}

	}
	
	
	public final static boolean containsDigit(final String s){    
	    for (char c: s.toCharArray()){
	        if(Character.isDigit(c)){
	            return true;
	        }
	    }

	    return false;
	}
	

}
